# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
from weibosearch.items import WeiboItem
import re
import time
import pymongo


# class WeibosearchPipeline(object):
# 数据清洗
class WeiboPipeline(object):
    def parse_item(self, datetime):
        if re.match('\d+月\d+日',datetime): #从开头处开始匹配
            datetime = time.strftime('%Y{y}', time.localtime()).format(y='年') + datetime
        elif re.match('\d+分钟前', datetime):
            minute = re.match('(\d+)', datetime).group(1)
            datetime = time.strftime('%Y{y}%m{m}%d{d} %H:%M', time.localtime(time.time() - float(minute)*60)).format(y='年', m='月', d='日')
        elif re.match('今天.*', datetime):
            datetime = re.match('今天(.*)', datetime).group(1).strip()
            # 错：datetime = time.strftime('%Y年%m月%d日', time.localtime()) + ' ' + datetime
            datetime = time.strftime('%Y{y}%m{m}%d{d}', time.localtime()).format(y='年', m='月', d='日') + ' ' + datetime
        return datetime

    def process_item(self, item, spider):
        if isinstance(item, WeiboItem):
            if item.get('content'):
                item['content'] = item['content'].lstrip(':').strip()
            if item.get('posted_at'):
                item['posted_at'] = item['posted_at'].strip()
                item['posted_at'] = self.parse_item(item['posted_at'])
        return item

# 数据存入MongoDB
class MongoPipeline():
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    # 定义类方法，类的两个参数通过crawler获得settings中的相关配置
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DB')
        )

    # open_spider()在spider开启的时候，会调用的方法，这里将它覆写
    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.colse()

    def process_item(self, item, spider):
        '''第一个参数，查询的字段，用id来查询——condition查询条件'''
        self.db[item.table_name].update({'id': item.get('id')}, {'$set': dict(item)}, True) #最后一个参数至关重要：True-如果查询到内容，更新，否则直接插入
        return item

